import re

import scrapy

from commonresources.spider_items.extra_demand.items_crawl_hang_ye_bai_qiang import convert_dict
from commonresources.spider_items.extra_demand.items_crawl_hang_ye_bai_qiang import HangYeBaiQiangItem


class HangYeBaiQiangSpider(scrapy.Spider):
    """
        行业百强    http://www.b2bname.com/product  https://www.itjuzi.com
    """
    name = 'HangYeBaiQiang'
    name_zh = '行业百强'

    def __init__(self, full_dose=True):
        self.convert_dict = convert_dict
        super().__init__()

    def parse(self, response):
        pass

    def start_requests(self):
        targets = [
          ["建筑工程", "地基与基础", "土方", "tufang", "http://www.b2bname.com/tufang/qiye", "https://www.itjuzi.com/search?data=%E5%9C%9F%E6%96%B9%E5%B7%A5%E7%A8%8B"],
          ["建筑工程", "地基与基础", "打桩", "dazhuang", "http://www.b2bname.com/product/dazhuang/qiye/", "https://www.itjuzi.com/search?data=%E5%9C%B0%E5%9F%BA"],
          ["建筑工程", "地基与基础", "回填", "huitian", "http://www.b2bname.com/product/huitian/qiye/", "/"],

          ["建筑工程", "主体结构", "混泥土框架", "nitukuangjia", "http://www.b2bname.com/product/nitukuangjia/qiye/", "/"],
          ["建筑工程", "主体结构", "墙体", "qt_538706", "http://www.b2bname.com/product/qt_538706/qiye/", "/"],
          ["建筑工程", "装修与装饰", "水电", "shuidiananzhuang", "http://www.b2bname.com/product/shuidiananzhuang/qiye/", "/"],

          ["建筑工程", "装修与装饰", "设备安装", "shebeianzhuang", "http://www.b2bname.com/product/shebeianzhuang/qiye/","https://www.itjuzi.com/search?data=%E8%AE%BE%E5%A4%87%E5%AE%89%E8%A3%85"],
          ["建筑工程", "装修与装饰", "消防", "xiaofanggongcheng", "http://www.b2bname.com/product/xiaofanggongcheng/qiye/", "https://www.itjuzi.com/search?data=%E6%B6%88%E9%98%B2%E5%B7%A5%E7%A8%8B"],
          ["建筑工程", "装修与装饰", "通风", "/", "https://www.itjuzi.com/search?data=%E9%80%9A%E9%A3%8E%E5%B7%A5%E7%A8%8B"],

          ["建筑工程", "装修与装饰", "装饰", "/", "https://www.itjuzi.com/search?data=%E8%A3%85%E4%BF%AE%E8%A3%85%E9%A5%B0"],
          ["土木工程", "道路工程", "道路工程", "https://www.itjuzi.com/search?data=%E9%81%93%E8%B7%AF%E5%B7%A5%E7%A8%8B", "https://www.itjuzi.com/search?data=%E9%81%93%E8%B7%AF%E5%B7%A5%E7%A8%8B"],
          ["土木工程", "轨道交通工程", "轨道交通工程", "guidaojiaotonggongcheng", "http://www.b2bname.com/product/guidaojiaotonggongcheng/qiye/", "https://www.itjuzi.com/search?data=%E8%BD%A8%E9%81%93%E4%BA%A4%E9%80%9A%E5%B7%A5%E7%A8%8B"],

          ["土木工程", "桥涵工程", "桥涵工程", "qhgc", "http://www.b2bname.com/product/qhgc/qiye/", "/"],
          ["土木工程", "隧道工程", "隧道工程", "sdgc", "http://www.b2bname.com/product/sdgc/qiye/", "https://www.itjuzi.com/search?data=%E9%9A%A7%E9%81%93%E5%B7%A5%E7%A8%8B"],
          ["土木工程", "矿山工程", "矿山工程", "kuangshangongcheng", "http://www.b2bname.com/product/kuangshangongcheng/qiye/", "/"],
        ]
        for target in targets:
            if "www.b2bname.com" in target[4]:
                url = f"http://www.b2bname.com/product/{target[3]}/qiye/"
                yield scrapy.Request(
                    url=url,
                    dont_filter=True,
                    headers={
                        "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
                        "Accept-Encoding":"gzip, deflate",
                        "Accept-Language":"zh-CN,zh;q=0.9",
                        "Cache-Control":"max-age=0",
                        "Connection":"keep-alive",
                        "Host":"www.b2bname.com",
                        "Upgrade-Insecure-Requests":"1",
                        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36",
                    },
                    callback=self.handle_response,
                    meta={
                        "工程分类": target[0],
                        "施工阶段": target[1],
                        "工序流程": target[2],
                        "page_count": -2,
                        "page": 1,
                        'o_url': url,
                    }
                )
            if "itjuzi.com/search?data=" in target[-1]:
                char_set = target[-1].split('=')[-1]
                url = f"https://www.itjuzi.com/api/search/result?word={char_set}&page=1&type[]=company"
                yield scrapy.Request(
                    url=url,
                    headers={
                        # ":method":"GET",
                        # ":path":"/api/search/result?word=%E5%9C%B0%E5%9F%BA&page=3&type[]=company",
                        # ":scheme":"https",
                        "accept": "application/json, text/plain, */*",
                        "accept-encoding": "gzip, deflate, br",
                        "accept-language": "zh-CN,zh;q=0.9",
                        "authorization": "bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwczpcL1wvd3d3Lml0anV6aS5jb21cL2FwaVwvYXV0aG9yaXphdGlvbnMiLCJpYXQiOjE2MDQ2NDMwODIsImV4cCI6MTYwNDY0NjY4MiwibmJmIjoxNjA0NjQzMDgyLCJqdGkiOiJxdkprTHNwRGIxVlc1cDZwIiwic3ViIjo4OTUxNDUsInBydiI6IjIzYmQ1Yzg5NDlmNjAwYWRiMzllNzAxYzQwMDg3MmRiN2E1OTc2ZjciLCJ1dWlkIjoicjlETGVnIn0.UwAtJYf7wA-TCD_9jezaeY9vvD9KVYHNWSTIcWYts6U",
                        "cookie": "_ga=GA1.2.259270427.1604642624; _gid=GA1.2.624222009.1604642624; Hm_lvt_1c587ad486cdb6b962e94fc2002edf89=1604642624; juzi_user=895145; juzi_token=bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwczpcL1wvd3d3Lml0anV6aS5jb21cL2FwaVwvYXV0aG9yaXphdGlvbnMiLCJpYXQiOjE2MDQ2NDMwODIsImV4cCI6MTYwNDY0NjY4MiwibmJmIjoxNjA0NjQzMDgyLCJqdGkiOiJxdkprTHNwRGIxVlc1cDZwIiwic3ViIjo4OTUxNDUsInBydiI6IjIzYmQ1Yzg5NDlmNjAwYWRiMzllNzAxYzQwMDg3MmRiN2E1OTc2ZjciLCJ1dWlkIjoicjlETGVnIn0.UwAtJYf7wA-TCD_9jezaeY9vvD9KVYHNWSTIcWYts6U; Hm_lpvt_1c587ad486cdb6b962e94fc2002edf89=1604643294",
                        "curlopt_followlocation": "true",
                        "dnt": "1",
                        "referer": "https://www.itjuzi.com/search?data=%E5%9C%B0%E5%9F%BA",
                        "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3823.400 QQBrowser/10.7.4307.400",
                    },
                    dont_filter=True,
                    callback=self.handle_response,
                    meta={
                        "工程分类": target[0],
                        "施工阶段": target[1],
                        "工序流程": target[2],
                        "page_count": -2,
                        "page": 1,
                        'o_url': f"https://www.itjuzi.com/api/search/result?word={char_set}&page=#&type[]=company",
                    }

                )

    def handle_response(self, response):
        if "b2bname.com" in response.url:
            if response.meta['page_count'] == -2:
                try:
                    response.meta['page_count'] = int(re.findall(r'qiye/(\d*).html">尾页', 'qiye/25.html">尾页')[0]) - 1
                except:
                    print("获取总页数失败")
                    response.meta['page_count'] = -1

            objs = response.xpath('//div[@class="block_piclist2 block_buy"]/li')
            for obj in objs:
                company_href = obj.xpath('./div[@class="l"]/a/@href').extract_first()
                company_name = obj.xpath('./div[@class="c"]/h3/a/text()').extract_first()
                company_area = obj.xpath('./div[@class="c"]/span/text()').extract_first().split("：")[-1]
                yield scrapy.Request(
                    url=company_href,
                    callback=self.handle_business_classification,
                    meta={
                        "企业名称": company_name,
                        "企业所在地区": company_area,
                        'origin_url': company_href,
                        '工程分类': response.meta['工程分类'],
                        '施工阶段': response.meta['施工阶段'],
                        '工序流程': response.meta['工序流程'],
                    },
                    dont_filter=True
                )

            page = response.meta['page']
            page_count = response.meta['page_count']
            if page < page_count:
                page += 1
                o_url = response.meta['o_url'] + str(page) + '.html'
                yield scrapy.Request(
                    url=o_url,
                    callback=self.handle_response,
                    dont_filter=True,
                    meta={
                        'o_url': response.meta['o_url'],
                        "page_count": page_count,
                        "page": page,
                        '工程分类': response.meta['工程分类'],
                        '施工阶段': response.meta['施工阶段'],
                        '工序流程': response.meta['工序流程'],
                    }
                )
        elif "itjuzi.com" in response.url:
            json_objs = response.json()['data']['company']
            if response.meta['page_count'] == -2:
                response.meta['page_count'] = json_objs['page']['total'] // 10 + 1
                # print(response.meta['page_count'])
            for obj in json_objs['data']:
                item = HangYeBaiQiangItem()
                item['business_classification'] = obj['des'].strip()
                item['company_name'] = obj['name'].strip()
                item['origin_url'] = str(obj['id'])+response.url
                item['company_area'] = None
                item['project_type_first'] = response.meta['工程分类']
                item['project_type_second'] = response.meta['施工阶段']
                item['project_type_third'] = response.meta['工序流程']
                yield item

            page = response.meta['page']
            page_count = response.meta['page_count']
            if page < page_count:
                page += 1
                o_url = response.meta['o_url'].replace("#", str(page))
                yield scrapy.Request(
                    url=o_url,
                    headers={
                        "accept": "application/json, text/plain, */*",
                        "accept-encoding": "gzip, deflate, br",
                        "accept-language": "zh-CN,zh;q=0.9",
                        "authorization": "bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwczpcL1wvd3d3Lml0anV6aS5jb21cL2FwaVwvYXV0aG9yaXphdGlvbnMiLCJpYXQiOjE2MDQ2NDMwODIsImV4cCI6MTYwNDY0NjY4MiwibmJmIjoxNjA0NjQzMDgyLCJqdGkiOiJxdkprTHNwRGIxVlc1cDZwIiwic3ViIjo4OTUxNDUsInBydiI6IjIzYmQ1Yzg5NDlmNjAwYWRiMzllNzAxYzQwMDg3MmRiN2E1OTc2ZjciLCJ1dWlkIjoicjlETGVnIn0.UwAtJYf7wA-TCD_9jezaeY9vvD9KVYHNWSTIcWYts6U",
                        "cookie": "_ga=GA1.2.259270427.1604642624; _gid=GA1.2.624222009.1604642624; Hm_lvt_1c587ad486cdb6b962e94fc2002edf89=1604642624; juzi_user=895145; juzi_token=bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwczpcL1wvd3d3Lml0anV6aS5jb21cL2FwaVwvYXV0aG9yaXphdGlvbnMiLCJpYXQiOjE2MDQ2NDMwODIsImV4cCI6MTYwNDY0NjY4MiwibmJmIjoxNjA0NjQzMDgyLCJqdGkiOiJxdkprTHNwRGIxVlc1cDZwIiwic3ViIjo4OTUxNDUsInBydiI6IjIzYmQ1Yzg5NDlmNjAwYWRiMzllNzAxYzQwMDg3MmRiN2E1OTc2ZjciLCJ1dWlkIjoicjlETGVnIn0.UwAtJYf7wA-TCD_9jezaeY9vvD9KVYHNWSTIcWYts6U; Hm_lpvt_1c587ad486cdb6b962e94fc2002edf89=1604643294",
                        "curlopt_followlocation": "true",
                        "dnt": "1",
                        "referer": "https://www.itjuzi.com/search?data=%E5%9C%B0%E5%9F%BA",
                        "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3823.400 QQBrowser/10.7.4307.400",
                    },
                    dont_filter=True,
                    callback=self.handle_response,
                    meta={
                        "工程分类": response.meta['工程分类'],
                        "施工阶段": response.meta['施工阶段'],
                        "工序流程": response.meta['工序流程'],
                        "page_count": page_count,
                        "page": page,
                        'o_url': response.meta['o_url'],
                    }
                )

    def handle_business_classification(self, response):
        business_classification = response.xpath('//div[@id="fd"]//ul/li[2]/text()').extract_first()
        if not business_classification:
            business_classification = response.xpath('//div[@class="block_body_text pub-fc"]/text()').extract_first()
        business_classification = business_classification.strip()
        item = HangYeBaiQiangItem()
        item['company_name'] = response.meta['企业名称']
        item['company_area'] = response.meta['企业所在地区']
        item['origin_url'] = response.meta['origin_url']
        item['project_type_first'] = response.meta['工程分类']
        item['project_type_second'] = response.meta['施工阶段']
        item['project_type_third'] = response.meta['工序流程']
        item['business_classification'] = business_classification
        yield item
